{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Test FileMonitor Cache Operations\n",
    "\n",
    "This notebook tests the file monitoring functionality of AutoCoderRAGDocListener, including:\n",
    "- File addition\n",
    "- File modification\n",
    "- File deletion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Created test directory: /var/folders/dm/0xljd5nn10b7bwwmwv8w5v100000gn/T/tmphovefknq\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import tempfile\n",
    "import shutil\n",
    "import time\n",
    "from autocoder.rag.variable_holder import VariableHolder\n",
    "from tokenizers import Tokenizer\n",
    "\n",
    "# Initialize tokenizer\n",
    "VariableHolder.TOKENIZER_PATH = \"/Users/allwefantasy/Downloads/tokenizer.json\"\n",
    "VariableHolder.TOKENIZER_MODEL = Tokenizer.from_file(VariableHolder.TOKENIZER_PATH)\n",
    "\n",
    "from autocoder.rag.cache.file_monitor_cache import AutoCoderRAGDocListener\n",
    "\n",
    "# Create a temporary directory for testing\n",
    "test_dir = tempfile.mkdtemp()\n",
    "print(f\"Created test directory: {test_dir}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m2024-12-17 14:54:54.246\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mautocoder.rag.cache.file_monitor_cache\u001b[0m:\u001b[36mopen_watch\u001b[0m:\u001b[36m90\u001b[0m - \u001b[1mstart monitor: /var/folders/dm/0xljd5nn10b7bwwmwv8w5v100000gn/T/tmphovefknq...\u001b[0m\n"
     ]
    }
   ],
   "source": [
    "# Initialize the cache manager\n",
    "cache_manager = AutoCoderRAGDocListener(\n",
    "    path=test_dir,\n",
    "    ignore_spec=None,\n",
    "    required_exts=[\".txt\", \".py\"]\n",
    ")\n",
    "\n",
    "# Give it some time to initialize\n",
    "time.sleep(2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\u001b[32m2024-12-17 14:55:06.092\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mautocoder.rag.utils\u001b[0m:\u001b[36mprocess_file_local\u001b[0m:\u001b[36m124\u001b[0m - \u001b[1mLoad file /private/var/folders/dm/0xljd5nn10b7bwwmwv8w5v100000gn/T/tmphovefknq/w1.txt in 0.0007789134979248047\u001b[0m\n",
      "\u001b[32m2024-12-17 14:55:06.093\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mautocoder.rag.cache.file_monitor_cache\u001b[0m:\u001b[36mupdate_cache\u001b[0m:\u001b[36m81\u001b[0m - \u001b[1mupdate cache: /private/var/folders/dm/0xljd5nn10b7bwwmwv8w5v100000gn/T/tmphovefknq/w1.txt\u001b[0m\n",
      "\u001b[32m2024-12-17 14:55:06.093\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mautocoder.rag.cache.file_monitor_cache\u001b[0m:\u001b[36mupdate_cache\u001b[0m:\u001b[36m82\u001b[0m - \u001b[1mcurrent cache: dict_keys(['/private/var/folders/dm/0xljd5nn10b7bwwmwv8w5v100000gn/T/tmphovefknq/w1.txt'])\u001b[0m\n",
      "\u001b[32m2024-12-17 14:55:15.598\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mautocoder.rag.utils\u001b[0m:\u001b[36mprocess_file_local\u001b[0m:\u001b[36m124\u001b[0m - \u001b[1mLoad file /private/var/folders/dm/0xljd5nn10b7bwwmwv8w5v100000gn/T/tmphovefknq/w1.txt in 0.0003781318664550781\u001b[0m\n",
      "\u001b[32m2024-12-17 14:55:15.599\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mautocoder.rag.cache.file_monitor_cache\u001b[0m:\u001b[36mupdate_cache\u001b[0m:\u001b[36m81\u001b[0m - \u001b[1mupdate cache: /private/var/folders/dm/0xljd5nn10b7bwwmwv8w5v100000gn/T/tmphovefknq/w1.txt\u001b[0m\n",
      "\u001b[32m2024-12-17 14:55:15.599\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mautocoder.rag.cache.file_monitor_cache\u001b[0m:\u001b[36mupdate_cache\u001b[0m:\u001b[36m82\u001b[0m - \u001b[1mcurrent cache: dict_keys(['/private/var/folders/dm/0xljd5nn10b7bwwmwv8w5v100000gn/T/tmphovefknq/w1.txt'])\u001b[0m\n",
      "\u001b[32m2024-12-17 14:55:35.691\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mautocoder.rag.cache.file_monitor_cache\u001b[0m:\u001b[36mremove_cache\u001b[0m:\u001b[36m86\u001b[0m - \u001b[1mremove cache: /private/var/folders/dm/0xljd5nn10b7bwwmwv8w5v100000gn/T/tmphovefknq/w1.txt\u001b[0m\n",
      "\u001b[32m2024-12-17 14:55:35.692\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mautocoder.rag.cache.file_monitor_cache\u001b[0m:\u001b[36mremove_cache\u001b[0m:\u001b[36m87\u001b[0m - \u001b[1mcurrent cache: dict_keys([])\u001b[0m\n",
      "\u001b[32m2024-12-17 14:55:55.214\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mautocoder.rag.utils\u001b[0m:\u001b[36mprocess_file_local\u001b[0m:\u001b[36m124\u001b[0m - \u001b[1mLoad file /private/var/folders/dm/0xljd5nn10b7bwwmwv8w5v100000gn/T/tmphovefknq/w.py in 0.00047016143798828125\u001b[0m\n",
      "\u001b[32m2024-12-17 14:55:55.215\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mautocoder.rag.cache.file_monitor_cache\u001b[0m:\u001b[36mupdate_cache\u001b[0m:\u001b[36m81\u001b[0m - \u001b[1mupdate cache: /private/var/folders/dm/0xljd5nn10b7bwwmwv8w5v100000gn/T/tmphovefknq/w.py\u001b[0m\n",
      "\u001b[32m2024-12-17 14:55:55.216\u001b[0m | \u001b[1mINFO    \u001b[0m | \u001b[36mautocoder.rag.cache.file_monitor_cache\u001b[0m:\u001b[36mupdate_cache\u001b[0m:\u001b[36m82\u001b[0m - \u001b[1mcurrent cache: dict_keys(['/private/var/folders/dm/0xljd5nn10b7bwwmwv8w5v100000gn/T/tmphovefknq/w.py'])\u001b[0m\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Cache after adding file:\n",
      "File: /private/var/folders/dm/0xljd5nn10b7bwwmwv8w5v100000gn/T/tmphovefknq/w1.txt\n",
      "Content: [{'module_name': '##File: /private/var/folders/dm/0xljd5nn10b7bwwmwv8w5v100000gn/T/tmphovefknq/w1.txt', 'source_code': 'This is a test file', 'tag': '', 'tokens': 13, 'metadata': {}}]\n"
     ]
    }
   ],
   "source": [
    "# Test file addition\n",
    "def test_file_addition():\n",
    "    # Create a test file\n",
    "    test_file = os.path.join(test_dir, \"w1.txt\")\n",
    "    with open(test_file, \"w\") as f:\n",
    "        f.write(\"This is a test file\")\n",
    "    \n",
    "    # Wait for the file monitor to detect the change\n",
    "    time.sleep(2)\n",
    "    \n",
    "    # Get cache and check if file is added\n",
    "    cache = cache_manager.get_cache()\n",
    "    print(\"\\nCache after adding file:\")\n",
    "    for file_path, data in cache.items():\n",
    "        print(f\"File: {file_path}\")\n",
    "        print(f\"Content: {data['content']}\")\n",
    "\n",
    "test_file_addition()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Cache after modifying file:\n",
      "File: /private/var/folders/dm/0xljd5nn10b7bwwmwv8w5v100000gn/T/tmphovefknq/w1.txt\n",
      "Content: [{'module_name': '##File: /private/var/folders/dm/0xljd5nn10b7bwwmwv8w5v100000gn/T/tmphovefknq/w1.txt', 'source_code': 'This is a modified test file', 'tag': '', 'tokens': 14, 'metadata': {}}]\n"
     ]
    }
   ],
   "source": [
    "# Test file modification\n",
    "def test_file_modification():\n",
    "    # Modify the test file\n",
    "    test_file = os.path.join(test_dir, \"w1.txt\")\n",
    "    with open(test_file, \"w\") as f:\n",
    "        f.write(\"This is a modified test file\")\n",
    "    \n",
    "    # Wait for the file monitor to detect the change\n",
    "    time.sleep(2)\n",
    "    \n",
    "    # Get cache and check if file content is updated\n",
    "    cache = cache_manager.get_cache()\n",
    "    print(\"\\nCache after modifying file:\")\n",
    "    for file_path, data in cache.items():\n",
    "        print(f\"File: {file_path}\")\n",
    "        print(f\"Content: {data['content']}\")\n",
    "\n",
    "test_file_modification()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Cache after deleting file:\n",
      "Files in cache: []\n"
     ]
    }
   ],
   "source": [
    "# Test file deletion\n",
    "def test_file_deletion():\n",
    "    # Delete the test file\n",
    "    test_file = os.path.join(test_dir, \"w1.txt\")\n",
    "    os.remove(test_file)\n",
    "    \n",
    "    # Wait for the file monitor to detect the change\n",
    "    time.sleep(2)\n",
    "    \n",
    "    # Get cache and verify file is removed\n",
    "    cache = cache_manager.get_cache()\n",
    "    print(\"\\nCache after deleting file:\")\n",
    "    print(f\"Files in cache: {list(cache.keys())}\")\n",
    "\n",
    "test_file_deletion()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Cache after adding Python file:\n",
      "File: /private/var/folders/dm/0xljd5nn10b7bwwmwv8w5v100000gn/T/tmphovefknq/w.py\n",
      "Content: [{'module_name': '##File: /private/var/folders/dm/0xljd5nn10b7bwwmwv8w5v100000gn/T/tmphovefknq/w.py', 'source_code': \"def hello():\\n    print('Hello World!')\", 'tag': '', 'tokens': 19, 'metadata': {}}]\n"
     ]
    }
   ],
   "source": [
    "# Test with Python files\n",
    "def test_python_file():\n",
    "    # Create a Python test file\n",
    "    test_file = os.path.join(test_dir, \"w.py\")\n",
    "    with open(test_file, \"w\") as f:\n",
    "        f.write(\"def hello():\\n    print('Hello World!')\")\n",
    "    \n",
    "    # Wait for the file monitor to detect the change\n",
    "    time.sleep(2)\n",
    "    \n",
    "    # Get cache and check if file is added\n",
    "    cache = cache_manager.get_cache()\n",
    "    print(\"\\nCache after adding Python file:\")\n",
    "    for file_path, data in cache.items():\n",
    "        print(f\"File: {file_path}\")\n",
    "        print(f\"Content: {data['content']}\")\n",
    "\n",
    "test_python_file()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Cleaned up test directory: /var/folders/dm/0xljd5nn10b7bwwmwv8w5v100000gn/T/tmphovefknq\n"
     ]
    }
   ],
   "source": [
    "# Cleanup\n",
    "def cleanup():\n",
    "    cache_manager.stop()\n",
    "    shutil.rmtree(test_dir)\n",
    "    print(f\"\\nCleaned up test directory: {test_dir}\")\n",
    "\n",
    "cleanup()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
